{
 "cells": [
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-19T05:34:57.060319Z",
     "start_time": "2025-03-19T05:34:53.731225Z"
    }
   },
   "source": [
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import sklearn\n",
    "import pandas as pd\n",
    "import os\n",
    "import sys\n",
    "import time\n",
    "from tqdm.auto import tqdm\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "print(sys.version_info)\n",
    "for module in mpl, np, pd, sklearn, torch:\n",
    "    print(module.__name__, module.__version__)\n",
    "    \n",
    "device = torch.device(\"cuda:0\") if torch.cuda.is_available() else torch.device(\"cpu\")\n",
    "print(device)\n",
    "\n",
    "seed = 42\n",
    "torch.manual_seed(seed)\n",
    "torch.cuda.manual_seed_all(seed)"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sys.version_info(major=3, minor=12, micro=3, releaselevel='final', serial=0)\n",
      "matplotlib 3.10.0\n",
      "numpy 2.0.2\n",
      "pandas 2.2.3\n",
      "sklearn 1.6.1\n",
      "torch 2.6.0+cu126\n",
      "cuda:0\n"
     ]
    }
   ],
   "execution_count": 1
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 数据准备"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-19T05:34:57.093862Z",
     "start_time": "2025-03-19T05:34:57.061318Z"
    }
   },
   "source": [
    "!wget https://storage.googleapis.com/download.tensorflow.org/data/shakespeare.txt"
   ],
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "'wget' 不是内部或外部命令，也不是可运行的程序\n",
      "或批处理文件。\n"
     ]
    }
   ],
   "execution_count": 2
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-19T05:36:38.991600Z",
     "start_time": "2025-03-19T05:36:38.986413Z"
    }
   },
   "source": [
    "# https://storage.googleapis.com/download.tensorflow.org/data/shakespeare.txt\n",
    "#文件已经下载好了\n",
    "with open(\"./shakespeare.txt\", \"r\", encoding=\"utf8\") as file:\n",
    "    text = file.read()\n",
    "\n",
    "print(\"length\", len(text))\n",
    "print(text[0:100])"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length 1115394\n",
      "First Citizen:\n",
      "Before we proceed any further, hear me speak.\n",
      "\n",
      "All:\n",
      "Speak, speak.\n",
      "\n",
      "First Citizen:\n",
      "You\n"
     ]
    }
   ],
   "execution_count": 4
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 构造字典"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-19T05:36:46.014495Z",
     "start_time": "2025-03-19T05:36:46.006666Z"
    }
   },
   "source": [
    "# 1. generate vocab\n",
    "# 2. build mapping char->id\n",
    "# 3. data -> id_data  把数据都转为id\n",
    "# 4. a b c d [EOS] -> [BOS] b c d  预测下一个字符生成的模型，也就是输入是a，输出就是b\n",
    "\n",
    "#去重，留下独立字符，并排序\n",
    "vocab = sorted(set(text))\n",
    "print(len(vocab))\n",
    "print(vocab)"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "65\n",
      "['\\n', ' ', '!', '$', '&', \"'\", ',', '-', '.', '3', ':', ';', '?', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z']\n"
     ]
    }
   ],
   "execution_count": 5
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-19T05:36:46.031616Z",
     "start_time": "2025-03-19T05:36:46.015495Z"
    }
   },
   "source": [
    "for idx,char in enumerate(['how','are','you']):\n",
    "    print(idx,char)"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 how\n",
      "1 are\n",
      "2 you\n"
     ]
    }
   ],
   "execution_count": 6
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-19T05:36:46.035314Z",
     "start_time": "2025-03-19T05:36:46.031616Z"
    }
   },
   "source": [
    "#每个字符都编好号，enumerate对每一个位置编号，生成的是列表中是元组，下面字典生成式\n",
    "char2idx = {char:idx for idx, char in enumerate(vocab)}\n",
    "print(char2idx)"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'\\n': 0, ' ': 1, '!': 2, '$': 3, '&': 4, \"'\": 5, ',': 6, '-': 7, '.': 8, '3': 9, ':': 10, ';': 11, '?': 12, 'A': 13, 'B': 14, 'C': 15, 'D': 16, 'E': 17, 'F': 18, 'G': 19, 'H': 20, 'I': 21, 'J': 22, 'K': 23, 'L': 24, 'M': 25, 'N': 26, 'O': 27, 'P': 28, 'Q': 29, 'R': 30, 'S': 31, 'T': 32, 'U': 33, 'V': 34, 'W': 35, 'X': 36, 'Y': 37, 'Z': 38, 'a': 39, 'b': 40, 'c': 41, 'd': 42, 'e': 43, 'f': 44, 'g': 45, 'h': 46, 'i': 47, 'j': 48, 'k': 49, 'l': 50, 'm': 51, 'n': 52, 'o': 53, 'p': 54, 'q': 55, 'r': 56, 's': 57, 't': 58, 'u': 59, 'v': 60, 'w': 61, 'x': 62, 'y': 63, 'z': 64}\n"
     ]
    }
   ],
   "execution_count": 7
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-19T05:36:46.039640Z",
     "start_time": "2025-03-19T05:36:46.036311Z"
    }
   },
   "source": [
    "# 把vocab从列表变为ndarray\n",
    "idx2char = np.array(vocab)\n",
    "print(idx2char)"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['\\n' ' ' '!' '$' '&' \"'\" ',' '-' '.' '3' ':' ';' '?' 'A' 'B' 'C' 'D' 'E'\n",
      " 'F' 'G' 'H' 'I' 'J' 'K' 'L' 'M' 'N' 'O' 'P' 'Q' 'R' 'S' 'T' 'U' 'V' 'W'\n",
      " 'X' 'Y' 'Z' 'a' 'b' 'c' 'd' 'e' 'f' 'g' 'h' 'i' 'j' 'k' 'l' 'm' 'n' 'o'\n",
      " 'p' 'q' 'r' 's' 't' 'u' 'v' 'w' 'x' 'y' 'z']\n"
     ]
    }
   ],
   "execution_count": 8
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-19T05:36:46.125198Z",
     "start_time": "2025-03-19T05:36:46.039640Z"
    }
   },
   "source": [
    "#把字符都转换为id\n",
    "text_as_int = np.array([char2idx[c] for c in text])\n",
    "print(text_as_int.shape)\n",
    "print(len(text_as_int))\n",
    "print(text_as_int[0:10])\n",
    "print(text[0:10])"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1115394,)\n",
      "1115394\n",
      "[18 47 56 57 58  1 15 47 58 47]\n",
      "First Citi\n"
     ]
    }
   ],
   "execution_count": 9
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 把莎士比亚文集分成一个一个的样本"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-19T05:36:46.130759Z",
     "start_time": "2025-03-19T05:36:46.125198Z"
    }
   },
   "source": [
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "class CharDataset(Dataset):\n",
    "    def __init__(self, text_as_int, seq_length):\n",
    "        self.sub_len = seq_length + 1\n",
    "        self.text_as_int = text_as_int\n",
    "        self.num_seq = len(text_as_int) // self.sub_len\n",
    "        \n",
    "    def __getitem__(self, index):\n",
    "        return self.text_as_int[index * self.sub_len: (index + 1) * self.sub_len]\n",
    "    \n",
    "    def __len__(self):\n",
    "        return self.num_seq\n",
    "    \n",
    "def collat_fct(batch):\n",
    "    src_list = []\n",
    "    trg_list = []\n",
    "    for part in batch:\n",
    "        src_list.append(part[:-1])\n",
    "        trg_list.append(part[1:])\n",
    "        \n",
    "    src_list = np.array(src_list)\n",
    "    trg_list = np.array(trg_list)\n",
    "    return torch.Tensor(src_list).to(dtype=torch.int64), torch.Tensor(trg_list).to(dtype=torch.int64)\n",
    "        \n",
    "\n",
    "train_ds = CharDataset(text_as_int, 100)\n",
    "train_dl = DataLoader(train_ds, batch_size=64, shuffle=True, collate_fn=collat_fct)"
   ],
   "outputs": [],
   "execution_count": 10
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 定义模型"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-19T05:36:46.234153Z",
     "start_time": "2025-03-19T05:36:46.131756Z"
    }
   },
   "source": [
    "class CharLSTM(nn.Module):\n",
    "    def __init__(self, vocab_size, embedding_dim=256, hidden_dim=1024):\n",
    "        super(CharLSTM, self).__init__()\n",
    "        self.embedding = nn.Embedding(vocab_size, embedding_dim)\n",
    "        self.lstm = nn.LSTM(embedding_dim, hidden_dim, batch_first=True)\n",
    "        self.fc = nn.Linear(hidden_dim, vocab_size)\n",
    "        \n",
    "    def forward(self, x, hidden=None):\n",
    "        x = self.embedding(x)\n",
    "        output, hidden = self.lstm(x, hidden)\n",
    "        x = self.fc(output)\n",
    "        return x, hidden\n",
    "    \n",
    "    \n",
    "vocab_size = len(vocab)\n",
    "sample_inputs = torch.randint(0, vocab_size, (2, 100))\n",
    "    \n",
    "print(\"{:=^80}\".format(\" 一层单向 LSTM \"))       \n",
    "for key, value in CharLSTM(vocab_size).named_parameters():\n",
    "    print(f\"{key:^40}paramerters num: {np.prod(value.shape)}\")\n",
    "    \n",
    "CharLSTM(vocab_size)(sample_inputs)[0].shape"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================== 一层单向 LSTM ===================================\n",
      "            embedding.weight            paramerters num: 16640\n",
      "           lstm.weight_ih_l0            paramerters num: 1048576\n",
      "           lstm.weight_hh_l0            paramerters num: 4194304\n",
      "            lstm.bias_ih_l0             paramerters num: 4096\n",
      "            lstm.bias_hh_l0             paramerters num: 4096\n",
      "               fc.weight                paramerters num: 66560\n",
      "                fc.bias                 paramerters num: 65\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "torch.Size([2, 100, 65])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 11
  },
  {
   "cell_type": "code",
   "source": [
    "4 * 1024*256"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2025-03-19T05:36:46.238382Z",
     "start_time": "2025-03-19T05:36:46.235152Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1048576"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 12
  },
  {
   "cell_type": "code",
   "source": [
    "4 * 1024*1024"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2025-03-19T05:36:46.242927Z",
     "start_time": "2025-03-19T05:36:46.239379Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4194304"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 13
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 训练"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-19T05:36:46.247767Z",
     "start_time": "2025-03-19T05:36:46.243925Z"
    }
   },
   "source": [
    "class SaveCheckpointsCallback:\n",
    "    def __init__(self, save_dir, save_step=5000, save_best_only=True):\n",
    "        \"\"\"\n",
    "        Save checkpoints each save_epoch epoch. \n",
    "        We save checkpoint by epoch in this implementation.\n",
    "        Usually, training scripts with pytorch evaluating model and save checkpoint by step.\n",
    "\n",
    "        Args:\n",
    "            save_dir (str): dir to save checkpoint\n",
    "            save_epoch (int, optional): the frequency to save checkpoint. Defaults to 1.\n",
    "            save_best_only (bool, optional): If True, only save the best model or save each model at every epoch.\n",
    "        \"\"\"\n",
    "        self.save_dir = save_dir\n",
    "        self.save_step = save_step\n",
    "        self.save_best_only = save_best_only\n",
    "        self.best_metrics = -1\n",
    "        \n",
    "        # mkdir\n",
    "        if not os.path.exists(self.save_dir):\n",
    "            os.mkdir(self.save_dir)\n",
    "        \n",
    "    def __call__(self, step, state_dict, metric=None):\n",
    "        if step % self.save_step > 0:\n",
    "            return\n",
    "        \n",
    "        if self.save_best_only:\n",
    "            assert metric is not None\n",
    "            if metric >= self.best_metrics:\n",
    "                # save checkpoints\n",
    "                torch.save(state_dict, os.path.join(self.save_dir, \"best.ckpt\"))\n",
    "                # update best metrics\n",
    "                self.best_metrics = metric\n",
    "        else:\n",
    "            torch.save(state_dict, os.path.join(self.save_dir, f\"{step}.ckpt\"))\n",
    "\n"
   ],
   "outputs": [],
   "execution_count": 14
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-19T05:43:27.214040Z",
     "start_time": "2025-03-19T05:36:46.247767Z"
    }
   },
   "source": [
    "# 训练\n",
    "def training(\n",
    "    model, \n",
    "    train_loader, \n",
    "    epoch, \n",
    "    loss_fct, \n",
    "    optimizer, \n",
    "    save_ckpt_callback=None,\n",
    "    stateful=False      # 想用stateful，batch里的数据就必须连续，不能打乱\n",
    "    ):\n",
    "    record_dict = {\n",
    "        \"train\": [],\n",
    "    }\n",
    "    \n",
    "    global_step = 0\n",
    "    model.train()\n",
    "    hidden = None\n",
    "    with tqdm(total=epoch * len(train_loader)) as pbar:\n",
    "        for epoch_id in range(epoch):\n",
    "            # training\n",
    "            for datas, labels in train_loader:\n",
    "                datas = datas.to(device)\n",
    "                labels = labels.to(device)\n",
    "                # 梯度清空\n",
    "                optimizer.zero_grad()\n",
    "                # 模型前向计算\n",
    "                logits, hidden = model(datas, hidden=hidden if stateful else None)\n",
    "                # 计算损失\n",
    "                loss = loss_fct(logits.reshape(-1, vocab_size), labels.reshape(-1))\n",
    "                # 梯度回传\n",
    "                loss.backward()\n",
    "                # 调整优化器，包括学习率的变动等\n",
    "                optimizer.step()\n",
    " \n",
    "                loss = loss.cpu().item()\n",
    "                # record\n",
    "                \n",
    "                record_dict[\"train\"].append({\n",
    "                    \"loss\": loss, \"step\": global_step\n",
    "                })\n",
    "   \n",
    "                # 保存模型权重 save model checkpoint\n",
    "                if save_ckpt_callback is not None:\n",
    "                    save_ckpt_callback(global_step, model.state_dict(), metric=-loss)\n",
    "                # udate step\n",
    "                global_step += 1\n",
    "                pbar.update(1)\n",
    "                pbar.set_postfix({\"epoch\": epoch_id})\n",
    "        \n",
    "    return record_dict\n",
    "        \n",
    "\n",
    "epoch = 100\n",
    "\n",
    "model = CharLSTM(vocab_size=vocab_size)\n",
    "\n",
    "# 1. 定义损失函数 采用交叉熵损失 \n",
    "loss_fct = nn.CrossEntropyLoss()\n",
    "# 2. 定义优化器 采用 adam\n",
    "# Optimizers specified in the torch.optim package\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n",
    "\n",
    "\n",
    "# save best\n",
    "if not os.path.exists(\"checkpoints\"):\n",
    "    os.makedirs(\"checkpoints\")\n",
    "save_ckpt_callback = SaveCheckpointsCallback(\"checkpoints/text_generation_lstm\", save_step=1000, save_best_only=True)\n",
    "\n",
    "\n",
    "model = model.to(device)\n",
    "record = training(\n",
    "    model, \n",
    "    train_dl, \n",
    "    epoch, \n",
    "    loss_fct, \n",
    "    optimizer, \n",
    "    save_ckpt_callback=save_ckpt_callback,\n",
    "    )"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "  0%|          | 0/17300 [00:00<?, ?it/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "f0e5726ab37a4817a8e074079c7b48e3"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "execution_count": 15
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-19T05:43:27.297415Z",
     "start_time": "2025-03-19T05:43:27.215036Z"
    }
   },
   "source": [
    "plt.plot([i[\"step\"] for i in record[\"train\"][::50]], [i[\"loss\"] for i in record[\"train\"][::50]], label=\"train\")\n",
    "plt.grid()\n",
    "plt.show()"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ],
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAikAAAGdCAYAAADXIOPgAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAUeFJREFUeJzt3Qd4FHX6wPE3CSEhkISeAAm9946AAiqd48SCigV7O/2fHh7e4Z2eyCme3Ts9xYJYDlE8QU+p0pGOdJEOoSShJyQhIST7f97f7mx2QxKFBHaS+X6eZ9nsZnaZfXcy8877KxPkcrlcAgAAYDPBgV4BAACAgpCkAAAAWyJJAQAAtkSSAgAAbIkkBQAA2BJJCgAAsCWSFAAAYEskKQAAwJbKSSmQm5srhw4dksjISAkKCgr06gAAgF9B54s9deqU1K5dW4KDg8tmkqIJSnx8fKBXAwAAXID9+/dLXFxc2UxStIJifcioqKgSe9/s7GyZM2eO9O/fX0JDQ8WpiAMxUMTAjTgQA0UMSiYOqamppshgHcfLZJJiNfFoglLSSUpERIR5T6dvhE6PAzEgBhbiQAwUMSjZOFxoVw06zgIAAFsiSQEAALZEkgIAAGyJJAUAANgSSQoAALAlkhQAAGBLJCkAAMCWSFIAAIAtkaQAAABbIkkBAAC2RJICAABsiSQFAADYkqOTlA+X7ZP/7gmWbUmnAr0qAAAgH0cnKTM2J8nipGDZf+J0oFcFAADk4+gkJdhz6WiXK9BrAgAA8nN4kuK+zyVLAQDAdhydpAR5KikkKQAA2I+jkxSrkkKOAgCA/Tg8SaGSAgCAXTk6SfHkKJJLjgIAgO04OknJG91DlgIAgN04PElx31NJAQDAfhydpAQJfVIAALArZycp1uieQK8IAAA4h6OTFPqkAABQRpOUF154wUyI9thjjxW53NSpU6V58+YSHh4ubdq0kRkzZogd0CcFAIAymKSsXr1aJkyYIG3bti1yuWXLlsmIESPknnvukXXr1smwYcPMbfPmzRJozDgLAEAZS1LS0tLk1ltvlffee0+qVKlS5LJvvPGGDBw4UEaPHi0tWrSQcePGSceOHeXNN9+UQKOSAgCAfZW7kBc9/PDDMmTIEOnbt6/8/e9/L3LZ5cuXy6hRo/yeGzBggEyfPr3Q12RlZZmbJTU11dxnZ2ebW8lxZydnz54t4fctXazPTgyIge+9UxEHYqCIQcnEobjxO+8kZcqUKfLjjz+a5p5fIykpSWJiYvye08f6fGHGjx8vY8eOPef5OXPmSEREhJSUw8laSAqWrVt/lhknt4rTzZ07V5yOGBADC3EgBooYFC8OGRkZcsmSlP3798ujjz5qVlY7wV4sY8aM8au+aCUlPj5e+vfvL1FRUSX2/8w5tV7WHTssTZs1k8GXNxSn0kxXv9N+/fpJaGioOBExIAYW4kAMFDEomThYLSGXJElZu3atHD582PQpseTk5MjixYtNHxNtogkJCfF7TWxsrCQnJ/s9p4/1+cKEhYWZW34aoJLcWIKD3V1ygoJDHL0RXqz4lkbEgBhYiAMxUMSgeHEobuzOq+Ps1VdfLZs2bZL169d7b507dzadaPXn/AmK6t69u8ybN8/vOc3K9PlAY54UAADs67wqKZGRkdK6dWu/5ypWrCjVqlXzPj9y5EipU6eO6VeitHmod+/e8sorr5jOttqnZc2aNfLuu+9KoDG6BwAAB804m5CQIImJid7HPXr0kMmTJ5ukpF27dvLll1+akT35k51AYJ4UAADK2BBkXwsXLizysRo+fLi52U1ec0+g1wQAAOTn8Gv3uO+ppAAAYD+OTlLymnsCvSYAACA/RycpVFIAALAvhycpDEEGAMCuHJ2keHIUmnsAALAhhycpjO4BAMCuHJ2kWH1SaO4BAMB+HJ6kMLoHAAC7cniS4r5ndA8AAPbj6CSFafEBALAvRycpeX1SAr0mAAAgP4cnKVRSAACwK0cnKcyTAgCAfTk7SRFmnAUAwK4cnaTkje4J9JoAAID8HJ6keCopQpYCAIDdODpJoU8KAAD25egkhasgAwBgXw5PUtz3VFIAALAfRycpzDgLAIB9OTpJCfZ8eiopAADYj7OTFKtPClkKAAC24+gkxdMlhUoKAAA25OwkhT4pAADYlqOTFO9VkAO9IgAA4BwOT1KYJwUAALtyeJLivqdPCgAA9uPoJIU+KQAA2Jejk5S85p5ArwkAAMjP4UmK+55KCgAA9uPoJIXmHgAA7MvhSYr7no6zAACU8iTl7bfflrZt20pUVJS5de/eXWbOnFno8pMmTTLVCt9beHi42G6eFCopAADYTrnzWTguLk5eeOEFadKkiTmwf/TRR3LNNdfIunXrpFWrVgW+RpOZbdu2ndPEYgd0nAUAoIwkKUOHDvV7/Nxzz5nqyooVKwpNUjQpiY2NFXv3SQn0mgAAgGIlKb5ycnJk6tSpkp6ebpp9CpOWlib16tWT3Nxc6dixozz//POFJjSWrKwsc7Okpqaa++zsbHMrKa7cHPdnyc0t0fctbazPTgyIge+9UxEHYqCIQcnEobjxC3KdZ4eMTZs2maQkMzNTKlWqJJMnT5bBgwcXuOzy5ctlx44dph9LSkqKvPzyy7J48WLZsmWLaToqzDPPPCNjx44953n9vyIiIqSk/Hg0SD7aESJNonLlkVa5Jfa+AABAJCMjQ2655RaTA2j3j4uepJw5c0YSEhLMf/jll1/K+++/L4sWLZKWLVv+qoyqRYsWMmLECBk3btx5VVLi4+Pl6NGjF/QhC/O/DQdl1JdbpEu9yjL53q7iVPq9zJ07V/r16yehoaHiRMSAGFiIAzFQxKBk4qDH7+rVq19wknLezT3ly5eXxo0bm587deokq1evljfeeEMmTJjwi6/VD9ihQwfZuXNnkcuFhYWZW0GvL8mNJbSc++NrlubkjfBixbc0IgbEwEIciIEiBsWLQ3FjV+x5UrSviW/V45f6sWhzUa1atcReQ5ADvSYAAKBYlZQxY8bIoEGDpG7dunLq1CnTR2ThwoUye/Zs8/uRI0dKnTp1ZPz48ebxs88+K5dddpmpvJw8eVJeeukl2bdvn9x7771iB0HCjLMAAJSJJOXw4cMmEUlMTJTo6GjTIVYTFG2rUtpXJTg4rzhz4sQJue+++yQpKUmqVKlimoeWLVv2q/qvXNpr9wR6TQAAQLGSlA8++KDI32tVxddrr71mbnYV5MlSXKZXCgAAsBNHX7uHPikAANiXw5MU+qQAAGBXjk5SvFdBZh43AABsx9FJSt4FBqmkAABgNw5PUtz3jO4BAMB+HJ6k0CcFAAC7cnSSYqGSAgCA/Tg6SaFPCgAA9uXwJMV9T4oCAID9ODxJoU8KAAB25egkxTtPCjkKAAC24+gkhT4pAADYF0kKlRQAAGzJ0UlKXnMPWQoAAHbj6CQlr7kn0GsCAADyc3SSQiUFAAD7cnSSknftHpIUAADsxtFJShDNPQAA2JajkxT6pAAAYF8OT1Lc9zT3AABgPw5PUpgnBQAAu3J0kmKN7mHGWQAA7MfRSQoXGAQAwL4cnqS472nuAQDAfhydpFhDkKmkAABgPw5PUtz35CgAANiPo5MU7zwpgV4RAABwDocnKe57mnsAALAfRycpvtPiMwwZAAB7cXSSYlVSFDkKAAD24vAkJS9LockHAAB7cXiSkvczc6UAAFCKk5S3335b2rZtK1FRUebWvXt3mTlzZpGvmTp1qjRv3lzCw8OlTZs2MmPGDLFbnxRFJQUAgFKcpMTFxckLL7wga9eulTVr1shVV10l11xzjWzZsqXA5ZctWyYjRoyQe+65R9atWyfDhg0zt82bN4sd+BRS6JMCAEBpTlKGDh0qgwcPliZNmkjTpk3lueeek0qVKsmKFSsKXP6NN96QgQMHyujRo6VFixYybtw46dixo7z55ptiB/RJAQDAvspd6AtzcnJMU056erpp9inI8uXLZdSoUX7PDRgwQKZPn17ke2dlZZmbJTU11dxnZ2ebW0nJycl7rzPZ2VI+2JmJihXTkoxtaUMMiIGFOBADRQxKJg7Fjd95JymbNm0ySUlmZqapokybNk1atmxZ4LJJSUkSExPj95w+1ueLMn78eBk7duw5z8+ZM0ciIiKkpJzNzQvB7NlzpMIFp2xlw9y5c8XpiAExsBAHYqCIQfHikJGRIcVx3oflZs2ayfr16yUlJUW+/PJLueOOO2TRokWFJioXYsyYMX4VGK2kxMfHS//+/U2H3ZJyWqs1KxeZn/v26yfRFULFiTTT1Q2wX79+EhpKDIiBc2OgiAMxUMSgZOJgtYRcsiSlfPny0rhxY/Nzp06dZPXq1abvyYQJE85ZNjY2VpKTk/2e08f6fFHCwsLMLT8NUEluLLk+445DQso5ekO8GPEtjYgBMbAQB2KgiEHx4lDc2BV7npTc3Fy//iO+tFlo3rx5fs9pRlZYH5ZLzaffLB1nAQCwmXLn2wwzaNAgqVu3rpw6dUomT54sCxculNmzZ5vfjxw5UurUqWP6lKhHH31UevfuLa+88ooMGTJEpkyZYoYuv/vuu2KXeVKCxCUuCWIyNwAASnOScvjwYZOIJCYmSnR0tJnYTRMUbatSCQkJEhycV5zp0aOHSWT++te/ypNPPmmGLuvIntatW4tdaDFF8xMuMAgAQClOUj744IMif69VlfyGDx9ubrblyVKopAAAYC+OvnaPsrql0CcFAAB7cXySYgWAFAUAAHtxfJJijfDxHY4MAAACjyTFc09rDwAA9kKSYlVSyFIAALAVkhTPPUkKAAD2QpLiraQEek0AAIAvkhTPPZO5AQBgLyQpnnsqKQAA2AtJCh1nAQCwJZIUzz05CgAA9kKSQiUFAABbIknx3JOjAABgL45PUoKppAAAYEuOT1KYzA0AAHsiSfHcMwQZAAB7IUnxZClM5gYAgL2QpHjuqaQAAGAvJCl0nAUAwJZIUjz35CgAANgLSYrnnj4pAADYC0mKt7kn0GsCAAB8kaR47umTAgCAvZCk0HEWAABbIknx3JOjAABgL45PUrh2DwAA9uT4JMVCx1kAAOzF8UkKHWcBALAnkhTvtXsCvSYAAMCX45MUKwBM5gYAgL04PklhMjcAAMpAkjJ+/Hjp0qWLREZGSs2aNWXYsGGybdu2Il8zadIkCQoK8ruFh4eLXQSJOzuhTwoAAKU4SVm0aJE8/PDDsmLFCpk7d65kZ2dL//79JT09vcjXRUVFSWJiove2b98+sQsmcwMAwJ7Knc/Cs2bNOqdKohWVtWvXSq9evQp9nVZPYmNjxY6YzA0AgDLYJyUlJcXcV61atcjl0tLSpF69ehIfHy/XXHONbNmyReyCIcgAAJSBSoqv3Nxceeyxx6Rnz57SunXrQpdr1qyZTJw4Udq2bWuSmpdffll69OhhEpW4uLgCX5OVlWVultTUVHOvzUt6Kyn6XlZzT/bZsyX63qWJ9bmd+vkVMSAGFuJADBQxKJk4FDd+Qa4LHHv70EMPycyZM2Xp0qWFJhuFrXCLFi1kxIgRMm7cuAKXeeaZZ2Ts2LHnPD958mSJiIiQkjRha7D8dDJYRjTKkctqUk0BAKCkZGRkyC233GKKFNo/9ZIkKY888oh8/fXXsnjxYmnQoMF5/6fDhw+XcuXKyWefffarKynaVHT06NEL+pBFJUzD/zVPtpwIlueHtZLhneqIE2kctCN0v379JDQ0VJyIGBADC3EgBooYlEwc9PhdvXr1C05Szqu5R/OZ//u//5Np06bJwoULLyhBycnJkU2bNsngwYMLXSYsLMzc8tMAlfTGYvVJCQ4OdvSGeLHiW9oQA2JgIQ7EQBGD4sWhuLE7ryRFhx9rk4tWUXSulKSkJPN8dHS0VKhQwfw8cuRIqVOnjplTRT377LNy2WWXSePGjeXkyZPy0ksvmSHI9957r9ir42yAVwQAAFx4kvL222+b+z59+vg9/+GHH8qdd95pfk5ISDBVCcuJEyfkvvvuMwlNlSpVpFOnTrJs2TJp2bKl2AHzpAAAYE/n3dzzS7QZyNdrr71mbnbFtXsAALAnrt3DtXsAALAlkhTPPc09AADYi+OTFAuVFAAA7MXxSYrV3EOfFAAA7MXxSUpex9kArwgAAPDj+CSFIcgAANgTSYrnnj4pAADYC0kKlRQAAGyJJMVzT8dZAADshSSFydwAALAlkhTPPc09AADYC0mK555KCgAA9kKSwmRuAADYEkmK557mHgAA7IUkxVtJCfSaAAAAXyQpnnv6pAAAYC+OT1Lyrt1DlgIAgJ04PklhxlkAAOyJJMVzT3MPAAD2QpJCJQUAAFsiSfHck6MAAGAvJCmeeyopAADYC0kKzT0AANgSSYq4kxNyFAAA7IUkxVtJCfSaAAAAXyQpnnsmcwMAwF5IUjxZSg6lFAAAbMXxSUqYJwIZZ3ICvSoAAMCH45OUCuXc96mZ2YFeFQAA4IMkxZOkpJwmSQEAwE5IUkLcfVFSSVIAALAVxycpEd7mnrOBXhUAAHChScr48eOlS5cuEhkZKTVr1pRhw4bJtm3bfvF1U6dOlebNm0t4eLi0adNGZsyYIXZRISSvuYdhyAAAlNIkZdGiRfLwww/LihUrZO7cuZKdnS39+/eX9PT0Ql+zbNkyGTFihNxzzz2ybt06k9jobfPmzWKnPik6BJkRPgAA2IfnEP3rzJo1y+/xpEmTTEVl7dq10qtXrwJf88Ybb8jAgQNl9OjR5vG4ceNMgvPmm2/KO++8I4FWPlgkNCRIsnNcZoRPxbDzCgkAALhIinVETklJMfdVq1YtdJnly5fLqFGj/J4bMGCATJ8+vdDXZGVlmZslNTXV3GvlRm8lRd9LJ3OLDCsnxzOy5VjqaaludVJxECumJRnb0oYYEAMLcSAGihiUTByKG78LPiLn5ubKY489Jj179pTWrVsXulxSUpLExMT4PaeP9fmi+r6MHTv2nOfnzJkjERERUtJCcs+YCfLnLFwiu6LEsbTC5XTEgBhYiAMxUMSgeHHIyMiQgCQp2jdF+5UsXbpUStqYMWP8qi9aSYmPjzf9X6KiSi6L0AxPAx9bNUqOHDolLdt1lqtb1BSnseLQr18/CQ0NFSciBsTAQhyIgSIGJRMHqyXkkiYpjzzyiHz77beyePFiiYuLK3LZ2NhYSU5O9ntOH+vzhQkLCzO3/DRAF2NjiY4ob+7Ts12O3hgvVnxLE2JADCzEgRgoYlC8OBQ3duc1ukeH6GqCMm3aNJk/f740aNDgF1/TvXt3mTdvnt9zmpXp83YRHe4OIlPjAwBgH+XOt4ln8uTJ8vXXX5u5Uqx+JdHR0VKhQgXz88iRI6VOnTqmX4l69NFHpXfv3vLKK6/IkCFDZMqUKbJmzRp59913xS4iPeOQmRofAAD7OK9Kyttvv21G9PTp00dq1arlvX3++efeZRISEiQxMdH7uEePHiax0aSkXbt28uWXX5qRPUV1tr3UosLdSUrqaWadBQCgVFZSfs2MrAsXLjznueHDh5ubXUXR3AMAgO04/to9KormHgAAbIckxbeSQpICAIBtkKRQSQEAwJZIUnwqKacy6TgLAIBdkKToEGpPJeVkhk6PDwAA7IAkRS+QaM04eyZHMrNzAr06AACAJMUtMrychIYEmZ+PpVNNAQDADkhS9PrHQUFStaK7mnI8jSQFAAA7IEnxqFrRfUHDY+lZgV4VAABAkpKneiVPJYXmHgAAbIEkxcNq7jlGcw8AALZAkpI/SaGSAgCALZCkeFSzOs7SJwUAAFsgSfGoVsnTcZbmHgAAbIEkxYPmHgAA7IUk5ZzmHpIUAADsgCTFwzuZG0kKAAC2QJKSr09KWtZZrt8DAIANkKR4RPlcv4dqCgAAgUeS4nP9niqeqyEzwgcAgMAjSfFRr1qEud+WfCrQqwIAgOORpPjoULeKuf8x4USgVwUAAMcjSfHRsW5lc78u4WSgVwUAAMcjSSmgkrItKdWM8gEAAIFDkuIjJipc6lSuILkukY0HqKYAABBIJCn5tKfJBwAAWyBJyad9nDtJ2XwwJdCrAgCAo5Gk5NOydpS533IoNdCrAgCAo5Gk5NPKk6QkHM+Q1MzsQK8OAACORZKST+WI8qbzrPqJagoAAAFDklIAmnwAAAg8kpQimny2HKLzLAAApSZJWbx4sQwdOlRq165tLso3ffr0IpdfuHChWS7/LSkpSeyqVe1oc09zDwAApShJSU9Pl3bt2slbb711Xq/btm2bJCYmem81a9YUu1dSdhxOk8zsnECvDgAAjlTufF8waNAgcztfmpRUruyeg8TuakWHS5WIUDmRkS3bk09JW8/cKQAAwMZJyoVq3769ZGVlSevWreWZZ56Rnj17FrqsLqc3S2qqu9klOzvb3EqK9V4FvWeLWpGybNdx2bj/hLSIqShlWVFxcApiQAwsxIEYKGJQMnEobvyCXC6X64JfHBQk06ZNk2HDhhXZzKP9Ujp37mwSj/fff18++eQTWblypXTs2LHA12gSM3bs2HOenzx5skRERMil8PW+YJl/KFh6xuTKjQ1zL8n/CQBAWZKRkSG33HKLpKSkSFSUuyuFrZKUgvTu3Vvq1q1rkpVfW0mJj4+Xo0ePXtCHLCrDmzt3rvTr109CQ0P9fve/jYkyauomaR8fLZ/e3UXCypXdgVBFxcEpiAExsBAHYqCIQcnEQY/f1atXv+Ak5ZI19/jq2rWrLF26tNDfh4WFmVt+GqCLsbEU9L5t46ua+/X7U6T12O/l03u6yeVNqktZdrHiW5oQA2JgIQ7EQBGD4sWhuLELSHlg/fr1UqtWLbGzBtUrSmRYXg43Y3NiQNcHAACnOe9KSlpamuzcudP7eM+ePSbpqFq1qmnCGTNmjBw8eFA+/vhj8/vXX39dGjRoIK1atZLMzEzTJ2X+/PkyZ84csbOQ4CD5920d5ZU522X9/pOyhasiAwBg7yRlzZo1cuWVV3ofjxo1ytzfcccdMmnSJDMHSkJCgvf3Z86ckccff9wkLtrptW3btvL999/7vYddXdGkhtStGiG9X1ooW5NOyScr9klOTq7c0aO+6Y8DAABslKT06dNHiuprq4mKryeeeMLcSitNUiLDy8mpzLPy1PTN5rmO9aowdwoAABdZ2R2yUkK0YtLaM02+5bNV+wO2PgAAOAVJyq/QqKb/ZG7frD8oaVlnA7Y+AAA4AUnKr9CxbhW/UT/pZ3LkfxsOBXSdAAAo60hSfoVr2teRJwc3lxm/v0Ju7hJvnpuyyt05+FhalqzddyLAawgAQNkTkMncShsdjnx/r0bm55pRYfLynG2y4UCKjJ+5VSavTDCdav/7UHfpVM89ARwAACg+KinnqXqlMOnfKtb8PGHRbpOgKKopAACULJKUC/BQ70ZSvVJ5aVg9r0Ptz0mnArpOAACUNTT3XIDWdaJlzV/7mZ9nb0mSBz5ZK9s8Scqhk6clMSVTOtXL62wLAADOH5WUYmoeG2nudxxOkzNnc+WW91bIDe8sM1PpAwCAC0eSUkzxVSIkonyISVAm/rBH9h7LEJ2Q98u1TPgGAEBxkKQUU3BwkDSJcVdTXpj5s/f57zYmmsQFAABcGJKUEtAsppL3Z73uYHSFUDmRkS1N/zpTnv7afb0fAABwfkhSSkCfZjXNfcXyIfLngc1lRNe63t/plZO1U632UcnNLfzCjAAAwB+je0rAoNaxsnzMVVKtYpiULxcsmdk50q1BVXn6m82y//hpGfD6YrPcK8PbyfWd4gK9ugAAlApUUkroSsm1oiuYBEWFh4bIlc1ryj+ub+u33Nyfkr0/u1wuWb7rmKRmZl/y9QUAoDQgSbmIujesJn1buJuC1Jp9x01yoj5YukdGvLdCnv9uawDXEAAA+yJJucgVlvfv6CLb/j5QwsoFy9G0MzJ5VYIs2n5E/u5JTqasZqgyAAAFoU/KJRBWLkQ61q0iy3cfk79MO3e0z6nMbIkMDw3IugEAYFdUUi6Rrg0Kv0LyxgMpZjp9qykIAABQSblkftO2lry7eLf0axkjvZvWMJ1oE1MzZcP+k3Lr+yvNMrWjw+UfN7SVK5rUCPTqAgAQcCQpl4jOSrvpmf5SLsRdvNKhyK/O3W6SFMuhlEzzHEkKAAA091xSVoJiaVU7yvtz/5YxEhIcJOsSTsquI2my71i6jPlqo7zx/Q7zMwAATkMlJYDaxkV7f356aEvJzsmVBduOyFc/HpDj6Wfks1XukT8zNyfKrMd6BXBNAQC49EhSAkgngHt/ZGeJCAuRuCoRpglIk5RvNhySyhXKe5f7OemU7Dx8ShrXdF/IEAAAJ6C5J8D6toyRHo2qm5+vbFbTNPnoVPo/Jaaa5xrWqGjuZ25KCuh6AgBwqZGk2EjFsHLSopa7WpKT65JKYeXkgV4NzeNX5m6X95fslvSsswFeSwAALg2SFJvpXC9vPpXmsZHSv2Wsqa4onaX2d//5Uc7m5AZwDQEAuDRIUmymc/0q3p9b1o6SKhXLy/PXtpYhbWtJhdAQM6X+a99v9y6jFyi8acJy+ePUDQFaYwAALg6SFBtXUlrWcg9RvqlLXXnrlo7y/HWtzeOvfjxoZqfVm06zv3LPcfly7QEzvT4AAGUFSYrNxEaHS9OYSqItPJ3q5VVV1KDWtaR8SLAkpmTK3mMZ8t8fD8r/Nhzy/n7XEeZTAQCUHQxBtqEP7ugiyamZZpZaX+GhIdKhbmVTOZm8cp/8Z2WCeV4TlzM5ubLrcJq0j68sy3YelZpR4dK4ZqUAfQIAAIqPSooNxVeNkM71C74goTVc+b0leyTjTI5c1rCq3NA5zjy380iaSVBueX+l6acCAICjkpTFixfL0KFDpXbt2hIUFCTTp0//xdcsXLhQOnbsKGFhYdK4cWOZNGnSha6v4/VoXM37c0xUmLx+Uwdp6qmY7DycJh8t32t+PpZ+RrLO5gRsPQEAuORJSnp6urRr107eeuutX7X8nj17ZMiQIXLllVfK+vXr5bHHHpN7771XZs+efSHr63jt4iqbqyVXjgiVj+/uZvqwWDPR/nQoVZbtOuZd9tDJzACuKQAAl7hPyqBBg8zt13rnnXekQYMG8sorr5jHLVq0kKVLl8prr70mAwYMON//3vHKlwuW2X/oJbkukegKoea5RjXds9IePHnab9kDJzKkQfWKkpvrkmDPXCsAAJQWF73j7PLly6Vv375+z2lyohWVwmRlZZmbJTXVPUV8dna2uZUU671K8j0vhfAQ8VvvahU8T+Sz90iahIhL/m/KBhnSJlaeGtK8TMWhJBEDYmAhDsRAEYOSiUNx43fRk5SkpCSJiYnxe04fa+Jx+vRpqVChwjmvGT9+vIwdO/ac5+fMmSMRERElvo5z586V0q5JVLDsSA2WrjVyJTRY5IfkYPls8Wb56aS7Re/jFQnSKWh3mY9DcREDYmAhDsRAEYPixSEjI0PK3BDkMWPGyKhRo7yPNaGJj4+X/v37S1SUe4KzkqAZnga+X79+EhrqbjoprTpfkSW7jqTJZQ2qysRl++SHWdu9CYrl8iv7SZSniaisxuFCEQNiYCEOxEARg5KJg9USYtskJTY2VpKTk/2e08eabBRURVE6Ckhv+WmALsbGcrHe91KqUzVU6lR1j/KpV63g+VESTmZJh6gIMzPtXR+uNvOufHx31zIVh+IiBsTAQhyIgSIGxYtDcWN30ZOU7t27y4wZM/ye06xMn8fFm2fFoqOAmsZEyqo9x80QZb3S8mtzt8uafSfM7xNTM6VmRVsW1AAADnfeR6e0tDTZuXOn3xBjHVpctWpVqVu3rmmqOXjwoHz88cfm9w8++KC8+eab8sQTT8jdd98t8+fPly+++EK+++67kv0k8Iqrkleh6lK/qplPRZOU0V9uPGfZfUfTpWbF6Eu8hgAAXIR5UtasWSMdOnQwN6V9R/Tnp59+2jxOTEyUhAT3dO1Khx9rQqLVE51fRYciv//++ww/voisoclKp8lvWN2/+UevDWTRawABAFAmKil9+vQxV98tTEGzyepr1q1bd/5rhwuiMwH/rk8jWbP3hIzsXk9+TDjp/Z1WVWY+2kvGffuTTFq2V/Yd46KEAAB7ojNCGfXEwLw5URpWd0/2poa0qS0hwUFSv5q738o+KikAAJviAoMOUKdyXh+VYR1qm/t61dyJy94iKimfrUqQZ77ZYmasBQDgUqOS4gA6Jf603/WQlNPZ0jausnmunqeS8nPSKRn95SY5czxI2p08LfVruPuzbDqQImO+2mR+HtqutnSqVyWAnwAA4ERUUhyiQ90q0qdZTe/juCoRYl3OZ/qGRJmxP0RueneVqZro7W/fbPa7BhAAAJcalRQHX6jQtxUnNMglyafcs9YeOHHar7OtPgYA4FKjkuJgA1vFmvv7r6gv9SLdz63dd0I+Xr7XbzkqKQCAQCBJcbBnh7WSiXd2lj/2ayL1I91llenrD8rC7UfMzw9f2civknI8/Ywkp2YGcI0BAE5CkuJgNSPD5armMWZelQaeJGXF7uOi0+Bc0aS6XN64hjdJOX0mR4b8c4n0e3WR6YALAMDFRpICo34l/2HGf+jX1Du9/sETp2XyqgRJTMmU1MyzsuVQSoDWEgDgJCQpMCqFijSu4Z475a6e9aVj3SpSKzrcTPx2JidXnvvuJ++y25NOBXBNAQBOwegeeL18QxvZdOiU3Ny1rnlcLiTYJCra3OM7EmhbclrgVhIA4BhUUuDVqnaU3N69voSG5G0W4aEh3p97NKpm7rclpZr7PUfTZfH2I7L/OKN/AAAljyQFv3pK/ad+09Lcb09Ok4RjGTL4jSUycuIqueLFBWboMgAAJYnmHhRp9IBmUjEsREb1ayp1q1aU0JAgScs6K/d/skZOZ+d4l5v/czJT5wMAShSVFBSpdZ1o+fetnaRxzUgzS23D6pW81/zRhOX+Xg3N49V7qaQAAEoWSQrOS+f6edWSvw5pKTd1iTc/b9h/UvYdS5d3F++Stxbs5MrJAIBio7kH5+XPg5pLv5Yx0j6+slSOKC8ul0uqVixvZqPt/dJC73Jt6kRLr6buyeDUmbO5phIDAMCvxVED5yUyPNRcTVkTFKWz1eqooPy2J+fNpTLphz3S7KmZpt8KAAC/FkkKik2TFlUprJzc2aO++XnXkXRzn5qZLc/87ycz1f6ERbsDup4AgNKF5h4U263d6kr5kCDp2zJGVu4+bp7bdSTNNAV9sGSPdzl6qQAAzgdJCopNJ3zTSeBUoxru0T+bD6bIiPdWmAsWWvYeTTd9U77bdEh2JKfJ/13VRCqUd08Wpx1tf9h11PR10SYlAABo7kGJaui5/k/GmRyToISVC5abOrtHAB0+lSXXvf2D/OHzDfLvhbtkyuoE7+uem7FVbv9glbw6d3vA1h0AYC8kKShRFcPKmev9WH5/dRP5xw1tpUZkmHm8+aB7Sn21zXOhwpTT2fLBUnez0Ic/7L3k6wwAsCeSFJS4Cj7X+xnWoY65b1jdXWHxteOw+0KFEz0JioqNyktwAADORpKCEle9krtq4nvtn0Y13X1V1O2X1TP3O5JPmb4oX6zZ7/1d8qlM028FAACSFJS4Z4e1km4NqspXv+vhfa5KRF5n2HsubyDBQTo8+ax8vzVZElMyJaJ8iAQFiRmqnJhyOkBrDgCwE5IUlLjmsVHy+QPdpWPdvCn0uzes7m0Kql+9otSr5m7+eXPBTnN/VfOa3pFB+4+fNonKP2b9LIdO+icsOUy3DwCOwRBkXBKXN6kuE+/sLM1i3bPTakKy52i6bDyQYh4Pal1L0rPOys7DabLnaJrc89FqyTqbK6cys+Xvw9qYZdbvPyl3TFwl13eMk6eHtgzo5wEAXHxUUnDJXNU8xttHpUlMXh+VyLBy0qdZDYmrEmEeP/X1FpOgqKU7jpr702dyZNhbP5iRQBN/yOtoCwAou0hSEBBt60Sb+5DgIHnvjs5m6HJ8VXcC40v7rag35u3wez4zO+cSrSkAIFBIUhAQ/VvFytu3dpSlf7pSLmtYzTxnVVJU6zpRpiOtXl1Zm3l8hymrhOMZ5ne//2yd/LDTXW0BAJQtJCkICK2gDGpTS2pF51VP4n2SlDGDWngf3//xGjmTkys9G1eTNp4KzO4j6fLn/26UbzYcklvfXxmATwAAsGWS8tZbb0n9+vUlPDxcunXrJqtWrSp02UmTJklQUJDfTV8H5Ne8VqTpmzKia7z0aFRNmnjmVtHp9NWTg1tIA8+kcHuPpcucn5IDur4AAJuN7vn8889l1KhR8s4775gE5fXXX5cBAwbItm3bpGbNmgW+JioqyvzeookKkF9oSLBMuqur93HjmEoy7+fD5ucrmlSXVrWjzfBlNW+rf4KiTT9VK5a/xGsMALBVJeXVV1+V++67T+666y5p2bKlSVYiIiJk4sSJhb5Gk5LY2FjvLSYmprjrDQeo75lLRY30XGXZml5/9d4TfsvuOuKeYj8/nb12/MytMmtz0kVdVwBAgCspZ86ckbVr18qYMWO8zwUHB0vfvn1l+fLlhb4uLS1N6tWrJ7m5udKxY0d5/vnnpVWrVoUun5WVZW6W1FT3Remys7PNraRY71WS71ka2TUOHePcc6pUCisnVzSqYtYvrnLelPu+tiemSLArV16cs10euKKBqbyodxbtlgmLdruXebZfoVU8u8bgUiIGbsSBGChiUDJxKG78glwunYj81zl06JDUqVNHli1bJt27d/c+/8QTT8iiRYtk5cpzOzBq8rJjxw5p27atpKSkyMsvvyyLFy+WLVu2SFxcXIH/zzPPPCNjx4495/nJkyebqg2cIyFNJLq8+6YyzoqMWe3OrRtH5UrtCJHFScHStmqubDzuLgzWDHfJXzrkiE5OO/bHEDl5xp2YjO14VgrJcQAAF0FGRobccsst5vivXT9sN+OsJjO+CU2PHj2kRYsWMmHCBBk3blyBr9FKjfZ78a2kxMfHS//+/S/oQxaV4c2dO1f69esnoaF515ZxmtIWh6WnN8qhlNPy7m0dZeaWJFn8zVZvgqIOZwZJm+59ZHtSmpxcsd77fGzLLnJVsxrnvJ/m6To9/8E9O2XcyL6lIgYXQ2nbDi4W4kAMFDEomThYLSEX6rySlOrVq0tISIgkJ/t3WtTH2tfk19AP2aFDB9m5033NloKEhYWZW0GvvRgby8V639KmtMThrds6eX9uFusekqyqVSwv0RVCZffRdPl6Q7JMW3fQ73U/J6VL2/gq8tT0zTKkbS25toO7krd233H55wKdhyVE/pwtUsPnYohOVFq2g4uNOBADRQyKF4fixu68Os6WL19eOnXqJPPmzfM+p/1M9LFvtaQoOTk5smnTJqlVq9b5ry2QT5OYSO/PL9/YTu65ooH5+Z/zd5gJ32pHh8sf+jY1z63bf0KG/usH+X7rYfnzfzd5X/fVj3nJzKZD7msJAUBZkXI6W5btPGqqxmV+dI82w7z33nvy0UcfydatW+Whhx6S9PR0M9pHjRw50q9j7bPPPitz5syR3bt3y48//ii33Xab7Nu3T+69996S/SRwJB12/M5tHWXC7Z3kymY1ZXDrWlI5IlT0b1H7yI6/vq10a1jVLLtw2xE5mubukG1dvFCn1//fhkPe99t8MNX8IU9emSBLdhzx+78OnjwtW0hiAJQyT3y5QW55f2WpnFvqvPuk3HTTTXLkyBF5+umnJSkpSdq3by+zZs3yDitOSEgwI34sJ06cMEOWddkqVaqYSox2vNXhy0BJGNg6rypXpWJ5Wfqnq2TX4TQJDw2RZrGRkppZcO/yLYdSZcHPh73XB1KbDqbKpyv2mYsclg8Jli3PDjDztyzbdVTu/WiNGdK8cHQfvyn8ASBQPlm+V1btPSEv3dDW7PPy0zmk5m097L1g64BWsfL+kt3y3aZEeff2zlIj0t6jCS6o4+wjjzxibgVZuHCh3+PXXnvN3IBLRYcst4uv7H0cFR5qptTXKsm/b+0ok5btlbk/JcuDn66VkxnuBGZYu1oyfUOizN16WJbtOmae06n4Nx1MkaoR5eXOD1ebBEXp72/sXHCSsudouqzZe1xu6BTHpIUALqrM7Bx5fsbPcjo7R4a2rWWuiZafzhF1Voc6epq8tYL8ypzt5jVfrNkvD1/ZWFbtOW72my1rl9zAlJLCtXvgCJ/c3U1WPnm19GxcXVrXdne2tRKUsb9tJc8MbeFdNv1M3hWWV+w+Jp+s2OdNUJT+QWfn5Mojk3+UBz5ZI2dz3L/TZa56ZaGM/nKjSYIKo01Go6du8DY9AUBBMs6clbQsd6X3rGef85dpm/z2T5psqM0H3U3Rr3+/3exfdB+lvtmQ1+dua+Ip+WLNAe9rNIHR5m0dTDD4n0vk6/X+gw3sgCQFjhAcHOQtheoVli16faCR3etJxbByElvBfbZRsXyI3OfpgKv9WL5ce8D8fGeP+t4k5bW52+XbjYkye0uyrNh93Dz/+eoE0xdGWeVV3QFYOwvLkH8ulalrD8hLs/IuFQHg0ss6m3dCcjElpWT67QfSs87KmK82yoc/+F/d3aL7jbcW7JTOf/9e+r+6yCy/ZOdRs8/5z8oEOXTytFluvueyIWrjwRRzuZDXv99h9i+Lth2RbzceMvsnLepGhpWTnFyXjPv2J+9rtFL86coE2ZZ8yuz3+jQr+NI2gUSSAsfRawBZ7urZwNssM6x+rtzcJU4WjO4j13WM8yYk2jO+TuUK8od+TSU4SMyooX8v3OV9D+14eywtS/45P29Y/YYDJ+X0mRy55b2V0uW5781OSm06kNfxdrOnE67ukLTd2NfGAyfl3o9Wy9p9/tP/A2XZdxsTTeWyuPRvb+XuY0WOZvl4+V5p+fRsmbpmv7dj/Muzt0mKp8KqsnJE/jF7u/l7/LX2H8+QXi8uMCcy1v6h+wvz5Omvt3jXTa/c/tmq/TL2fz+Zx+qnQ6lmP2KNOHxp9jbJOJMjh1IyZeWeYzLNZxTi6r3HzWfzTVJW7j7u/T/Uv+bvkCe+3Gh+vr9XQ+nRuJr3d6EhQdKilvtkTaso6pZudc0UDnZDkgLHiYkKkyFtasllDavKdR3reJ9vUdkl437bUmpGhkuzmEip4jNfyoO9G5o/YOsCh6ppjPsqzZ+v2W/6rBw5lSWR4e5uXnpmcv8na2T57mOmWWnm5kTzvHbKtehOUc/k7vlojXQcN9fseNTOw6fkt2+6h0rr2ZTl8KnMUjmEEGXPwm2HZXvyqRJ9T23WeHTKOnPQ1L5dxTH6yw1y07sr5Ov1eSP3fGnTiFYUtLKgHUjVqM/Xm0kdJ/pUN+YdCpb3l+71HvxPpJ+Rv3/7k+w8XPC1wpT2edMTGb0/mXFGxv5vi6mw/nftAfP61+dtl/X785KedQknzPr85l9L5Np/LzNNPL7roGZuSpLZW/KuP6b7Cu34f+DEaSlfLljKBQeZJhzdp2jFRG04kGKSnO4Nq8no/s2kU70q3tc/f20b+V2fRt7H+vq7L3dXj+2GJAWOo5WTt27tKFPu715gb3ireei5a9vIjZ3j5JN7usrtngsc3tg53tw/2LuRzPj9FSbhscqmUeHlZNrvekqjGhXNTmnJjqPe99Mznh3Jp/wmmNPk5eZ3V3jPhvSMS5OQR6fkzZK7wbMz02Sl63PzZOoad9NTYXz7zgD5aZWguImu9rfSpPyuD1eb99L3vP/jNfLZqoQiX6dNDz/szPubyE87nFsdPLWqUFRnUT0YF1XJsBKPGZ57X1q5+P2UdZKd4+lMmnDSVExX7nGfJOjB3+oDsuKwu8qqlRRNOF6du13eX7rHJB4F0Sad6Z6/ca3Aauf8o2lnvB3x31m8Sz5attc8ruDZ96zae9xUTvSja3Jz96TVZh3CygWbZEJp841OmxCipVxNUvac8Ma7X8sYqRyRdwX48de7X2N55cZ2Ui4kWG7qXFduv6yeTL63mwzvHC9D29WWLx7obp578Ya2Uiu6gtgRSQpQiMFtasmLN7STK5rkTaWvycnGZ/rLnwc1N3/4v7+6iWkK6tsiRj69t5s0rllJujV0l1V1fzJmUHPzsyYsj0xeZ3ZUVzWv6R19pDtIy/Jdx8zIIWsnqY6ln5Gtianyz3k7zOOpa92l6a9+PCBXvbzQb94WPdNq/cxsv+pLfrqj1Z2glrp9z+R8S9za+1/P6nTIdVEHtAmLdsn4GVup7pQSeibf7tk5MvEH93ev35ue2fvS53QGZj3AKt0WdTuzmkRyc13mquJKE4V9xzLko+V7zfwb2jyhvy+Invnr9q/bXmFTAlhJgtJtXtdFqx36/1nbmCYoN7yzTHq/uMCbyGiF8eekvL8ZTQKsTVKTIk3cdQqBv3292TS7PvvtFtl9JN2cYGgioJ/1SZ/OqPp/q0Xbj0qK57pf+rH0M+rfnfW3qn8zCccy5PkZW031U+kJh/7NWqz+agM9o270YqeZ2bnSoW5leXJIC2/n11meSqvva67tUMdUfH2NHtDMW6nVvinq1m51pUt9d5VEP4++5q9DWpjqyD9HdJDald3JR3REqIwb1lp6NHZffFV1bVDVPGc1b9sRSQpwnnRIs+XWbvXkhz9fJe/f0VnaxrkTj1u61jXtvS8Pb2fagutVi/DuWHSY39+HtZZWPkP9buocbzq27TicZnZ4SjvzWq/TM1Y9i1LaR0V3oqO+2GCm///XPHdCcjg1U/70341mh/xfz4500fYjct/Ha0wSYvlg6R6zI9W2dy11a29+LTGPmeZuu1ZvL9pjhmt/vzVZdh0puKytO/3xM3+WCYt3mypSQbYlnZJ9x/zL9ruPpJnXFkXPavXgpAckpQeoC0mE9ICpMdBOh9ZZ7oJthy9ZtUkP9taIi8L8Z+U+eXVO3sFdD+C+CeOvpWfV2g/C6vO0ePsR03RivZc2K2oSYSUrSvt+dBg3V6atcz/WdfjbN1vk+reXy2NfuLeHF2ZtM9uZJr76HWhyqwd4iybVVh8S7Vel27DGWYe26gHcYiXOuh3P3ZLsrWj4fq96sLZo0jFjU5LZXvXAbm2H+veh26ZWXHQ71ySk7yuLTGd0fY3+HUxZ7U6otOqgI/Wmrz8o9320Rj5avk96v7TA9AVRr97YXtrGufunafONp0hhki/9HiYtd3+u8FD3YVL7d1gj//T/1xOG2z5YKe8u3i3D31luEo1n/+fulOo794g2C784vK1Ur+Suduj/86eBzeWyBlW9SYn2O9GOq4/1bSKXN65upjDQPnDREaHSsIa7iVn3KQ/0aigNfZqc9WdtztGTphFd42X2Y71MpfjeKxrKT88OlN+2qy2lHUkKUMJa14mWmY9eYc5OdIehFRlVvVKYvHNbJ3Nm095nHpe//KaFtPR0YtMzV01Y7rm8gXkflZTq7nSrOzk9lg16Y4n3tdq2rTt6TVCsIdV6EHn6681yx8RVpjRvHSC0rds6oOgEdlqFsTrW6QFBD1LHMsW7c1ZzfzpsDiYvzPzZlOuVe4SAO5lSq/eeMOugZ9vWEMa9R9Nl6JtL5apXFsm/5u0wv9c+B7oz106D1pmvHhB8EyE9q71xwnJzcNJZf5UmZHowtUY0KD1z1b4LehZdGB2KqTGwzpK1I6MmfFYlwPr/xny1Sb73GTKePyHSpNBKmAqjv7/23z+Y70Z/1mY6/az6WTRR0KaDl2b/7B0ppiYu3SN/mbbZdLheuP2wOcDqAbfva4vMd2XR11qdK5V+T74VC02G9DNoU8EHS/VMPUcen7rB9MeY7GkS+GL1fu929FNiqombxlhpU4N+Zk1QPvZ890t3HpNlyUGyYo87wd17LENenrNNxnpGhmjHS/XMN1tMXyzfREOHyOp29Ycv3M2WGgsdJWfRdevz0gJp8fQsuf2DVeb/1kTSt1O5Jr76//luhz8mnPCun9L11+9Xt2XdJj9cutesn25n7eKivQdoK7nQvytNkrQPh1YPdDqCDnXz+mlc076OuYyGee8le8xnDwlyyZiB7uqFJa6KuzLx9++2mpirExnatPOj2Z7rV4sw8zFZ7u7ZwJzYzP9jH7Nf0JOayxpWM1VXnTHbcnWLGHmsb1NTkdUTnJgo97q8eH1b06n147u7mv3JX4a0MJUTXY8/DmhmnqtXraKMv66tX585/ZxlwUW/CjLgdHrtoE51q0iXBlW9veeHta9jqgza9KM7MD0bspp5dHnd6bSpE21GO6i+LWqaM6l/+YwgUnrg0RECC7YdMTul6hXLm7My3535Uk/J+/PV+72JjNIzVKtCo3YdTZdFicGmrV7by7Ujnp6h6xmidsLTM8peTWvIjI2J3pK41ZdAD4x6pq4Hgm4NqpkKgVWxeGXudmkbX1k27j/pLYXr8EiNxaDXF5v/b/ETV5qzT9+mKq24aLOY1Y9H+xfoGaL+X/1fW2wStrO5uWbnrGfCmhhpPLU/kZ7Fv7N4t7evz+/6NPYesDUOepaqcf/zfzfKzM1J5nPuGT/YdHS+/+O10qtmkAwWkSmrEuTPX22S+KoV5MXr20n3RtXMgejGd5ZLj0bV5KXh7cx7vjhrm7fpTg/6+n66ftpxUftH6Jn5Wwt2mYN7twZVzegw3z4cnyzfZ87OD3sO+Jr4XdG4uoz77if5cs0Byc7Nlf8+1MMks9rPQZtPvtU+UZFh8ofPN+R9F/tOmETIShw0hlrN04qXVV3QA7o2H2pTjVW50rhrAqvfn5Wjfb7b3WfCek7XX2kfhj7NapgO39p8qWKjws22qImORat+2iTzyGc/msed61Ux62clPda2+eb8nfLF2v3m82sirn04klN1/fOSH63qWZep0ORDE2b39+8yHdw1SdAO7NZnfP66NqZTqbXtRJQPkSn3XyZr9p4w27AmCKpj3byTBe1Imno62/z9vOFpXu0Z45Jr29eWCYv3mPW6qUu8uWkCrjHRbVYvyzFl1X7zN6jNLZPu6mqqoBojXa9hHdyd83V7i6qVV4XV5GLcNa1NE5JOgaDbZEE6169qbr7JjN6cgiQFuMg0eejbMuac50YPcPdXUTd3jTfNMDd0jpOHert73VuTzik9wyoXEmQOJtUqhZkzLT0T1+YMHUWgnhjQTBJT8s6QNcnRxzppnC6nBwOlTU2aEFmdCy1adl51xH2G/LehLc3BWc8UEzxdBbQt/fNV+2XCYvfBalDrWHOAt25Kd9yaTOiEUVb1SP9/Pdj7dprUg9MPu455L0lgjRZ5b0neqAY9eGZmu4dxKk0CdFrvKavdCYDSJO5U5lkzf4TStnitQv3tm83eJEmXHfD6Yu/7aOKglQXd8VvrbVWlRk/daM7ENVnTCoZVedp//LTcNWmV/O+Ry+X9JXtMoqKdGbVPUnJqpt9oDN/+Deqprzeb/gFKE7Jhb/3gTdasGOoBzpdeDC7x5Gn50NN/RP17wS659bK63uuv6EixtnWizXekCaR+P3pg/qtnSKkVQ52wS5/Xa1ppcqzby6cr8hIkTVRfnuOO81NDWkqTmEqmwqH0bF2bNP8x62fzWA+8OvmhxtyifTu0c6ZWyPK7+d3l5jvWJE+bRLXyoQmvNk0obXrRJNa9rZQ3SZ9WuyyP92tqfm8Nxdck74mBzcx3qImLWvDHPjJy4irZ6KnE/LF/MzPNgCZ0k+/rZipZzWtFSYPqFb1NshZNWK5o4q6o6MVKm9eKlHmejuwVw0KkX52zUqF8iMz+Qy9x+TT1aiKiyXLvpjVM5/tO9fKSCIsu80v0aux6Q+FIUgAbaFwz0pSDfWmnNu0E16hmJW/Tj+6QNUnRvi3av0OTD/WbtrVMWVl3sFaScvfl9WXJ9qPy1bqDpm+K1YatPfm1DV/pwe3yxjXMDn/8rG2SnRMkcZXDzSgmfZ0e5LSa0bhGJdMH4DlPnxl9Hz2g+B7kLc96mgS0Q/H469qYA4i1nHWm7DvyySqdW501ddI8/b/04KbJjEWTqvyJVapPgqLeW+KuGJjKUkiwPDW0pXceCNU8NlJ+TjplKlLW6ArL/Z+s9VYg0s8Gyaer9nurW3rA02YSjaMe8C1albEOlvpd+a6fJkz6ubRyoTdNErSSpQmK5iwf3tXVHORGvLvCVHCsDpaztriTPmumUR2FodUgHcbu2zSmiZZ24FT3Xt7QjBLR70vVjAwzyZkmiP/nqWTod6oX4bSSWqUHbmu479XNa5phqNqUNLBVjOw9kCgf3t9VIsLDTDOSnu1rPw6tVGlfCe07of2dPrijizSNiTRDX09lnTXfn1YStMKn349uYxNu62xGoLw7srO5rpZVkdJkVmOjlcQP7uwsEeXLyZXNapjvT6cBeOSqxjJ3a7I3AXmgVyNz3Sztg6ExfKhPI/O+zw1rYxJ4TfatCcm0UtGjUV4n0YLo//fJPd0KnEPpzZvbS+p2d+IV6dMPTen3hkuDJAWwKa226FBpX9oMZLm+U5zpRKjNB8M7u/u/6NwvWr3Q4dDaFyZIgkyyYXliYHNzwNVy94ETGfLhnV3NaB490FpDMnXYtR6ItP+MHsA6xFeWo+lZ3g65WhXQzr+aKNWKDjfVGvXNIz3N/C6WB3o3NAcj6+CstLytZ/pWJaF/yxhTGbASFB0N9UDvRuZgbCUy2lnww2V7vZWRrvWrmgRMO0hafR20eqJVFW1y0MRA6YFMmyY0FlZfkFeGt5MHPl1rqjL6f2qnRE0uNGmxEpQ2daLMhSafm7HNu446HF37m1jNFFYTg86robTpSvs56PtpR0odOqrfjx5cv9lwSIa1r236D2gzlVYBNKG0DnSaNGp16IZO8aZZQpMUq6+Dfk+v3tjONENoQqoJllYT9P/TZE/jqN/H7d3rmWYOK0nR99Q+KdrcocmKNtnc1q2eqWhoc6L2SdHqQa7L5e2XpH0dlH73/7q5ncyYcdBsS6GhobJo9JUSHBRkqgqWD+/qYj6L1YQ58a4ucjg1Swa3iTXNNdr5Wf9vHSFnXRNG+1lYfS002Xjmt61k84EU+etvWpiEQT05uIVJyvU71W36pRvayZwtSWZbsub60LhowmNpExct79zeSYpLR+mZic8aVZOeDavIjLxCHgKEJAUopbT0rGV2X3rGN+/x3uaAFVYuRK5qUdM07+jB4u6e9WVga/dQyOkP9zTlaqs5xtIw0iW3X1bX/Kyd+qyOfTrBnVYGtOz+eP+mpvJjJR3vLtktz17Typz5WvRgqAdFPeDpjl+TBL0cgf7/WiXQA7cecHSIpE5kpwc7PfCP9MxHo50YNUnRC0NqYqX9N6wk5bWb25sqzZG0MyZJ0QO79ifQDsl6YNQDsg7LvKtn3nvpzaJnznptE00EJtzeyTSTWH19NKm5qll1ueujtd7l9exc+x58/UhPMweGNu/oeg5/Z5lJVPT/e+o3LU2sHu/fzDQB6ZWz1UvD25rfWaM9tLL0o4lhXmfM+KoRfk1/+j7WDMRv3NzevJcmXNqvpXKFULm2Y5wkpZw2F5bTRPaP/ZuaA78mqjqSRT+rVhPO5rhMkqJJhDaR1PWMFnu0bxPv/6UdcrUZTisZDWu4+2kURKso+el6RVfI65zZxaffRGx0uLnCrvYZ8r1KeX4a7/y02cU3Pnolc71dChpPTZJUdvb5j7JCySNJAcoY36mtNZH57vdXnLOMVkH0pjRR0SpAakaW1ErZ6n0+P+20em++t9K+Nr79bV67qZ1JSF64rq1JUNSjVzcxw1K1NK9nxprkaD+Gey5vaNrzdZSFjkLSA5Z1pn59xzqmyUCHiGrC9bsrG5sRRjpcWxMUdV2HOrJ273G5snlN0wR2V4/6pplDD2ja/FAY/Xxv35Z31q2jS7SaciQty92M4MqRlpVzpVzFynJ/70ZyVXP359P/V68Ya5l832WmOUg7X/oe4K0ERWmiWCMyr/qQP2EqyJ8HNjd9XNxD1d3ND9ppWicK9F3nAa1iTTKg/4dVmdCKh+93M//x3mYZq0qRn8b7jZs7yMWQvx8WcCFIUgCYBEHPHGd4+pxcqGs7xJmbL60U+B4ItcnqL0Naeh9rR0xt+rBm81WazPiemWvpX5uptOTve3b/us/7alKk/TcudAZiS3a2Sx5okSuDB19mmjoKo4mDdf2TknRjl3hzK4o17PSXFFUdAUoDkhQAAaXNNLcVUPb3pdUJHYkBwFnKxmwvAACgzCFJAQAAtkSSAgAAbIkkBQAA2BJJCgAAsCWSFAAAYEskKQAAwJZIUgAAgC2RpAAAAFsiSQEAALZEkgIAAGyJJAUAANgSSQoAALClUnEVZJfLZe5TU1NL9H310vQZGRnmfYu6JHtZRxyIgSIGbsSBGChiUDJxsI7b1nG8TCYpp06dMvfx8fGBXhUAAHABx/Ho6OjzfZkEuS40vbmEcnNz5dChQxIZGSlBQUEl9r6a4Wnis3//fomKihKnIg7EQBEDN+JADBQxKJk4aIqhCUrt2rUlODi4bFZS9IPFxcVdtPfXwDt5I7QQB2KgiIEbcSAGihgUPw4XUkGx0HEWAADYEkkKAACwJUcnKWFhYfK3v/3N3DsZcSAGihi4EQdioIiBPeJQKjrOAgAA53F0JQUAANgXSQoAALAlkhQAAGBLJCkAAMCWHJ2kvPXWW1K/fn0JDw+Xbt26yapVq6Q0Gj9+vHTp0sXMyFuzZk0ZNmyYbNu2zW+ZPn36mNl6fW8PPvig3zIJCQkyZMgQiYiIMO8zevRoOXv2rN8yCxculI4dO5qe3o0bN5ZJkyaJXTzzzDPnfMbmzZt7f5+ZmSkPP/ywVKtWTSpVqiTXX3+9JCcnl6kY6PacPwZ6089dVreDxYsXy9ChQ82Mlvp5pk+f7vd7HRvw9NNPS61ataRChQrSt29f2bFjh98yx48fl1tvvdVMVlW5cmW55557JC0tzW+ZjRs3yhVXXGH2FzoD54svvnjOukydOtVsc7pMmzZtZMaMGWKHOOj1V/70pz+ZdapYsaJZZuTIkWYm71/afl544YVSE4df2hbuvPPOcz7fwIEDy9S2sPgXYlDQ/kFvL730kj23A5dDTZkyxVW+fHnXxIkTXVu2bHHdd999rsqVK7uSk5Ndpc2AAQNcH374oWvz5s2u9evXuwYPHuyqW7euKy0tzbtM7969zWdMTEz03lJSUry/P3v2rKt169auvn37utatW+eaMWOGq3r16q4xY8Z4l9m9e7crIiLCNWrUKNdPP/3k+te//uUKCQlxzZo1y2UHf/vb31ytWrXy+4xHjhzx/v7BBx90xcfHu+bNm+das2aN67LLLnP16NGjTMXg8OHDfp9/7ty5OnrPtWDBgjK7Heg6/uUvf3F99dVX5rNOmzbN7/cvvPCCKzo62jV9+nTXhg0bXL/97W9dDRo0cJ0+fdq7zMCBA13t2rVzrVixwrVkyRJX48aNXSNGjPD+XmMUExPjuvXWW83f2WeffeaqUKGCa8KECd5lfvjhBxOHF1980cTlr3/9qys0NNS1adOmgMfh5MmT5jv9/PPPXT///LNr+fLlrq5du7o6derk9x716tVzPfvss37bh+9+xO5x+KVt4Y477jDfte/nO378uN8ypX1bmPELMfD97HrTY2BQUJBr165dttwOHJuk6B/oww8/7H2ck5Pjql27tmv8+PGu0k4PVLpxLlq0yPucHpweffTRIjfs4OBgV1JSkve5t99+2xUVFeXKysoyj5944gmTBPi66aabTJJklyRFdy4F0Z20/oFMnTrV+9zWrVtNnHSHXVZikJ9+540aNXLl5uY6YjvIv1PWzx0bG+t66aWX/LaFsLAws2NVugPV161evdq7zMyZM82O++DBg+bxv//9b1eVKlW8MVB/+tOfXM2aNfM+vvHGG11DhgzxW59u3bq5HnjgAdelVtDBKb9Vq1aZ5fbt2+d3cHrttdcKfU1pikNhSco111xT6GvK2rYgv2I70HhcddVVfs/ZaTtwZHPPmTNnZO3atabs63t9IH28fPlyKe1SUlLMfdWqVf2e/89//iPVq1eX1q1by5gxY8zlty36ubUcFxMT431uwIAB5uJSW7Zs8S7jGzNrGTvFTMv4WuZs2LChKdlq04XS71tL3r7rr2XIunXrete/rMTAdzv/9NNP5e677/a7MKcTtgPLnj17JCkpyW999Toi2rzr+71rWb9z587eZXR53SesXLnSu0yvXr2kfPnyfp9Zm1VPnDhR6uJi7Sd0u9DP7kvL+tok2qFDB9ME4NvUVxbioE2V2ozZrFkzeeihh+TYsWPe3zltW0hOTpbvvvvONGnlZ5ftoFRcYLCkHT16VHJycvx2xEof//zzz1Ka6RWjH3vsMenZs6c5CFluueUWqVevnjmAa1uitk/rBvXVV1+Z3+uOvKB4WL8rahk9gJ0+fdq09weSHni0b4TufBITE2Xs2LGmzXTz5s1m3fUPKv8OWdf/lz6f9bvSEANf2hZ98uRJ0w7vpO3Al7XOBa2v7+fRg5avcuXKmSTfd5kGDRqc8x7W76pUqVJoXKz3sBPtn6Xf/YgRI/wuGvf73//e9DXSz75s2TKTxOrf0quvvlom4qD9T6677jrzGXbt2iVPPvmkDBo0yBw4Q0JCHLctfPTRR6Yvo8bEl522A0cmKWWZdpDUg/LSpUv9nr///vu9P+uZsnYivPrqq80faqNGjaQs0J2NpW3btiZp0QPyF198YasD56XywQcfmJhoQuKk7QBF04rijTfeaDoUv/32236/GzVqlN/fkCb2DzzwgOmcXxamh7/55pv9tn/9jLrda3VF/w6cZuLEiabirB1b7bodOLK5R0vdmjXnH9mhj2NjY6W0euSRR+Tbb7+VBQsWSFxcXJHL6gFc7dy509zr5y4oHtbvilpGz8TsmARo1aRp06bmM+q6a/OHVhYK+87LUgz27dsn33//vdx7772O3g6sdS7qb13vDx8+7Pd7LW3rKI+S2DbstE+xEhTdPubOnetXRSls+9BY7N27t0zFwaLNwno88N3+nbItLFmyxFRRf2kfEejtwJFJimaFnTp1knnz5vk1k+jj7t27S2mjZ0SaoEybNk3mz59/ThmuIOvXrzf3eiat9HNv2rTJ7w/U2om1bNnSu4xvzKxl7BozHTaoFQL9jPp9h4aG+q2//oFqnxVr/ctSDD788ENTttahxE7eDvRvQXeKvuurzVLav8D3e9fkVfstWfTvSPcJVhKny+jQTj3I+35mbVrU0nZpiIuVoGi/LU1gtb/BL9HtQ/tjWE0gZSEOvg4cOGD6pPhu/07YFqxKq+4X27VrJ7beDlwOHoKsPfwnTZpkenTff//9Zgiy76iG0uKhhx4yQywXLlzoN2QsIyPD/H7nzp1mOJkOu92zZ4/r66+/djVs2NDVq1evc4ae9u/f3wxj1uGkNWrUKHDo6ejRo83ImLfeestWw28ff/xxEwP9jDr8TYdc6vBZHe1kDUHWodnz5883sejevbu5laUYWCPV9HNqb3tfZXU7OHXqlBkurTfdpb366qvmZ2vUig5B1r9t/bwbN240oxkKGoLcoUMH18qVK11Lly51NWnSxG/YqY4I0iGXt99+uxlyqfsPjUH+IZflypVzvfzyyyYuOtrsUg5BLioOZ86cMUOv4+LizPfqu5+wRmgsW7bMjOjQ3+tw1E8//dR89yNHjiw1cSgqBvq7P/7xj2Y0n27/33//vatjx47mu87MzCwz28KpX/h7sIYQ6zrryL387LYdODZJUTq/g+7Mdb4UHZKs4+JLI90QC7rp3CkqISHBHIiqVq1qEjMd968HGN/5MdTevXtdgwYNMuPd9eCuB/3s7Gy/ZXS+jfbt25uY6QHO+j/sQIfB1qpVy6xbnTp1zGM9MFv0oPS73/3ODJ3TP6hrr73W7KTLUgzU7Nmzzfe/bds2v+fL6nag61LQ9q/DTa1hyE899ZTZqernvvrqq8+JzbFjx8yBqFKlSma49V133WV29r50jpXLL7/cvIduX5r85PfFF1+4mjZtauKiw7S/++47lx3ioAflwvYT1hw6a9euNUNE9YQnPDzc1aJFC9fzzz/vdwC3exyKioGetGnyrQdcPVjqMFudMyj/iWlp3xYW/MLfg9JkQv++NdnIz27bQZD+c361FwAAgIvPkX1SAACA/ZGkAAAAWyJJAQAAtkSSAgAAbIkkBQAA2BJJCgAAsCWSFAAAYEskKQAAwJZIUgAAgC2RpAAAAFsiSQEAALZEkgIAAMSO/h8SqzYIRLnZAgAAAABJRU5ErkJggg=="
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "execution_count": 16
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 推理"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-19T05:43:28.618609Z",
     "start_time": "2025-03-19T05:43:27.298412Z"
    }
   },
   "source": [
    "def generate_text(model, start_string, max_len=1000, temperature=1.0, stream=True):\n",
    "    input_eval = torch.Tensor([char2idx[char] for char in start_string]).to(dtype=torch.int64, device=device).reshape(1, -1)\n",
    "    hidden = None\n",
    "    text_generated = []\n",
    "    model.eval()\n",
    "    pbar = tqdm(range(max_len))\n",
    "    print(start_string, end=\"\")\n",
    "    with torch.no_grad():\n",
    "        for i in pbar:\n",
    "            logits, hidden = model(input_eval, hidden=hidden)\n",
    "            # 温度采样\n",
    "            logits = logits[0, -1, :] / temperature\n",
    "            # using multinomial to sampling\n",
    "            probs = F.softmax(logits, dim=-1)\n",
    "            idx = torch.multinomial(probs, 1).item()\n",
    "            input_eval = torch.Tensor([idx]).to(dtype=torch.int64, device=device).reshape(1, -1)\n",
    "            text_generated.append(idx)\n",
    "            if stream:\n",
    "                print(idx2char[idx], end=\"\", flush=True)\n",
    "    return \"\".join([idx2char[i] for i in text_generated])\n",
    "\n",
    "\n",
    "torch.manual_seed(seed)\n",
    "torch.cuda.manual_seed_all(seed)\n",
    "# load checkpoints\n",
    "model.load_state_dict(torch.load(\"checkpoints/text_generation_lstm/best.ckpt\", map_location=\"cpu\"))\n",
    "start_string = \"All: \"\n",
    "res = generate_text(model, start_string, max_len=1000, temperature=0.5, stream=True)"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "  0%|          | 0/1000 [00:00<?, ?it/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "434a929257334b2dbdeaadb9cfb6baeb"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All: for them now we were as good a sea,--\n",
      "\n",
      "SAMILLO:\n",
      "Besides, he tells me, holy sir; and so\n",
      "The king's will for earth to make this cap;\n",
      "And, she after him I cannot brook down\n",
      "to please me when I am arrived for me and honour,\n",
      "Purchase the king and serves the blood, and then\n",
      "Let us be pent.\n",
      "\n",
      "LADY ANNE:\n",
      "Fouler than I was for that, the change of York\n",
      "Usect another room and fable in arms,\n",
      "Be he my wretched bootled and summon\n",
      "Are disturb'd in the world but undertakings,\n",
      "Nearted the present compassion of her secret\n",
      "Than the son's safety mothers' house, that heaven\n",
      "By ears a crumpent, and so stuff'd his deeds,\n",
      "I will presently to put a toy, as it\n",
      "perform'd, which I do lose a thousand with our houses!\n",
      "\n",
      "LEONTES:\n",
      "No, no; I am not any more.\n",
      "\n",
      "POMPEY:\n",
      "But I can give not to you; in sadness punish me\n",
      "For I will soon at all that here was drown'd.\n",
      "\n",
      "GREMIO:\n",
      "A bridegroom say I do not know that you\n",
      "Have too much blood in that are goner,\n",
      "But much Frunk, call thee with a humble lord.\n",
      "\n",
      "KING RICHARD II:\n",
      "To your wil"
     ]
    }
   ],
   "execution_count": 17
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
